{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests,bs4\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getHtml(url, header):\n",
    "    #发起请求，获取网页内容源代码\n",
    "    try:\n",
    "        # 发送HTTP GET请求\n",
    "        r = requests.get(url, headers=header)\n",
    "        # 如果请求失败，抛出HTTPError异常\n",
    "        r.raise_for_status()\n",
    "        # 打印请求头信息\n",
    "        # print(r.request.headers)\n",
    "        # 返回网页内容\n",
    "        return r.text\n",
    "    except:\n",
    "        # 如果发生异常，打印爬取失败信息\n",
    "        print('爬取失败')\n",
    "        return \" \""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ParsePage(ulist, html):\n",
    "    # 解析源代码，获取有用的信息，并存入列表ulist\n",
    "    soup = bs4.BeautifulSoup(html, 'html.parser')\n",
    "    # 查找帖子区域的容器\n",
    "    container = soup.find('div', {'class': 'bbs-search-web-content'})\n",
    "    if container:\n",
    "        # 遍历帖子区域的所有子节点\n",
    "        for i in container.children:\n",
    "            try:\n",
    "                # 判断classname是否为'content-outline'\n",
    "                if i.name == 'div' and i.attrs['class'][0] == 'content-outline':\n",
    "                    # 获取所有class为'content-wrap-span'的子节点\n",
    "                    group1 = i.find_all('a', {'class': 'content-wrap-span'})\n",
    "                    # 获取所有标签为span的子节点\n",
    "                    group2 = i.find_all('span')\n",
    "                    # 将标题，专区，发布时间，回复数，推荐数，亮评数存入列表\n",
    "                    ulist.append([group1[0].text, group1[0]['href'], group1[1].text, group2[0].text, group2[1].text, group2[2].text, group2[3].text])\n",
    "            except:\n",
    "                # 如果发生异常，打印数据丢失信息\n",
    "                print(\"数据丢失！\")\n",
    "    else:\n",
    "        print(\"未找到帖子区域的容器\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main(ulist, keyWord):\n",
    "    header = {\n",
    "        \"user-agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36\",\n",
    "    }\n",
    "    for i in range(1, 11):\n",
    "        url = f'https://bbs.hupu.com/search?q={keyWord}&topicId=&sortby=general&page={i}'\n",
    "        print(url)\n",
    "        html = getHtml(url, header)\n",
    "        ParsePage(ulist, html)\n",
    "    # 存储为Excel文件\n",
    "    df = pd.DataFrame(ulist, columns=['Title', 'Link', 'Section', 'Post Time', 'Replies', 'Recommendations', 'Highlights'])\n",
    "    df.to_excel(f'{keyWord}热门帖.xlsx', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://bbs.hupu.com/search?q=ChatGPT&topicId=&sortby=general&page=1\n",
      "https://bbs.hupu.com/search?q=ChatGPT&topicId=&sortby=general&page=2\n",
      "https://bbs.hupu.com/search?q=ChatGPT&topicId=&sortby=general&page=3\n",
      "https://bbs.hupu.com/search?q=ChatGPT&topicId=&sortby=general&page=4\n",
      "https://bbs.hupu.com/search?q=ChatGPT&topicId=&sortby=general&page=5\n",
      "https://bbs.hupu.com/search?q=ChatGPT&topicId=&sortby=general&page=6\n",
      "https://bbs.hupu.com/search?q=ChatGPT&topicId=&sortby=general&page=7\n",
      "https://bbs.hupu.com/search?q=ChatGPT&topicId=&sortby=general&page=8\n",
      "https://bbs.hupu.com/search?q=ChatGPT&topicId=&sortby=general&page=9\n",
      "https://bbs.hupu.com/search?q=ChatGPT&topicId=&sortby=general&page=10\n",
      "https://bbs.hupu.com/search?q=Claude&topicId=&sortby=general&page=1\n",
      "https://bbs.hupu.com/search?q=Claude&topicId=&sortby=general&page=2\n",
      "https://bbs.hupu.com/search?q=Claude&topicId=&sortby=general&page=3\n",
      "https://bbs.hupu.com/search?q=Claude&topicId=&sortby=general&page=4\n",
      "https://bbs.hupu.com/search?q=Claude&topicId=&sortby=general&page=5\n",
      "https://bbs.hupu.com/search?q=Claude&topicId=&sortby=general&page=6\n",
      "https://bbs.hupu.com/search?q=Claude&topicId=&sortby=general&page=7\n",
      "https://bbs.hupu.com/search?q=Claude&topicId=&sortby=general&page=8\n",
      "https://bbs.hupu.com/search?q=Claude&topicId=&sortby=general&page=9\n",
      "https://bbs.hupu.com/search?q=Claude&topicId=&sortby=general&page=10\n",
      "数据丢失！\n",
      "https://bbs.hupu.com/search?q=Copilot&topicId=&sortby=general&page=1\n",
      "https://bbs.hupu.com/search?q=Copilot&topicId=&sortby=general&page=2\n",
      "https://bbs.hupu.com/search?q=Copilot&topicId=&sortby=general&page=3\n",
      "https://bbs.hupu.com/search?q=Copilot&topicId=&sortby=general&page=4\n",
      "https://bbs.hupu.com/search?q=Copilot&topicId=&sortby=general&page=5\n",
      "数据丢失！\n",
      "https://bbs.hupu.com/search?q=Copilot&topicId=&sortby=general&page=6\n",
      "数据丢失！\n",
      "https://bbs.hupu.com/search?q=Copilot&topicId=&sortby=general&page=7\n",
      "数据丢失！\n",
      "https://bbs.hupu.com/search?q=Copilot&topicId=&sortby=general&page=8\n",
      "数据丢失！\n",
      "https://bbs.hupu.com/search?q=Copilot&topicId=&sortby=general&page=9\n",
      "数据丢失！\n",
      "https://bbs.hupu.com/search?q=Copilot&topicId=&sortby=general&page=10\n",
      "数据丢失！\n",
      "https://bbs.hupu.com/search?q=Gemini&topicId=&sortby=general&page=1\n",
      "https://bbs.hupu.com/search?q=Gemini&topicId=&sortby=general&page=2\n",
      "https://bbs.hupu.com/search?q=Gemini&topicId=&sortby=general&page=3\n",
      "https://bbs.hupu.com/search?q=Gemini&topicId=&sortby=general&page=4\n",
      "https://bbs.hupu.com/search?q=Gemini&topicId=&sortby=general&page=5\n",
      "https://bbs.hupu.com/search?q=Gemini&topicId=&sortby=general&page=6\n",
      "https://bbs.hupu.com/search?q=Gemini&topicId=&sortby=general&page=7\n",
      "https://bbs.hupu.com/search?q=Gemini&topicId=&sortby=general&page=8\n",
      "https://bbs.hupu.com/search?q=Gemini&topicId=&sortby=general&page=9\n",
      "https://bbs.hupu.com/search?q=Gemini&topicId=&sortby=general&page=10\n",
      "https://bbs.hupu.com/search?q=DeepSeek&topicId=&sortby=general&page=1\n",
      "https://bbs.hupu.com/search?q=DeepSeek&topicId=&sortby=general&page=2\n",
      "https://bbs.hupu.com/search?q=DeepSeek&topicId=&sortby=general&page=3\n",
      "https://bbs.hupu.com/search?q=DeepSeek&topicId=&sortby=general&page=4\n",
      "https://bbs.hupu.com/search?q=DeepSeek&topicId=&sortby=general&page=5\n",
      "https://bbs.hupu.com/search?q=DeepSeek&topicId=&sortby=general&page=6\n",
      "https://bbs.hupu.com/search?q=DeepSeek&topicId=&sortby=general&page=7\n",
      "https://bbs.hupu.com/search?q=DeepSeek&topicId=&sortby=general&page=8\n",
      "https://bbs.hupu.com/search?q=DeepSeek&topicId=&sortby=general&page=9\n",
      "https://bbs.hupu.com/search?q=DeepSeek&topicId=&sortby=general&page=10\n",
      "https://bbs.hupu.com/search?q=豆包&topicId=&sortby=general&page=1\n",
      "https://bbs.hupu.com/search?q=豆包&topicId=&sortby=general&page=2\n",
      "https://bbs.hupu.com/search?q=豆包&topicId=&sortby=general&page=3\n",
      "https://bbs.hupu.com/search?q=豆包&topicId=&sortby=general&page=4\n",
      "https://bbs.hupu.com/search?q=豆包&topicId=&sortby=general&page=5\n",
      "https://bbs.hupu.com/search?q=豆包&topicId=&sortby=general&page=6\n",
      "https://bbs.hupu.com/search?q=豆包&topicId=&sortby=general&page=7\n",
      "https://bbs.hupu.com/search?q=豆包&topicId=&sortby=general&page=8\n",
      "https://bbs.hupu.com/search?q=豆包&topicId=&sortby=general&page=9\n",
      "https://bbs.hupu.com/search?q=豆包&topicId=&sortby=general&page=10\n",
      "https://bbs.hupu.com/search?q=Kimi&topicId=&sortby=general&page=1\n",
      "https://bbs.hupu.com/search?q=Kimi&topicId=&sortby=general&page=2\n",
      "https://bbs.hupu.com/search?q=Kimi&topicId=&sortby=general&page=3\n",
      "https://bbs.hupu.com/search?q=Kimi&topicId=&sortby=general&page=4\n",
      "https://bbs.hupu.com/search?q=Kimi&topicId=&sortby=general&page=5\n",
      "https://bbs.hupu.com/search?q=Kimi&topicId=&sortby=general&page=6\n",
      "https://bbs.hupu.com/search?q=Kimi&topicId=&sortby=general&page=7\n",
      "https://bbs.hupu.com/search?q=Kimi&topicId=&sortby=general&page=8\n",
      "https://bbs.hupu.com/search?q=Kimi&topicId=&sortby=general&page=9\n",
      "https://bbs.hupu.com/search?q=Kimi&topicId=&sortby=general&page=10\n",
      "https://bbs.hupu.com/search?q=文心一言&topicId=&sortby=general&page=1\n",
      "https://bbs.hupu.com/search?q=文心一言&topicId=&sortby=general&page=2\n",
      "https://bbs.hupu.com/search?q=文心一言&topicId=&sortby=general&page=3\n",
      "https://bbs.hupu.com/search?q=文心一言&topicId=&sortby=general&page=4\n",
      "https://bbs.hupu.com/search?q=文心一言&topicId=&sortby=general&page=5\n",
      "https://bbs.hupu.com/search?q=文心一言&topicId=&sortby=general&page=6\n",
      "https://bbs.hupu.com/search?q=文心一言&topicId=&sortby=general&page=7\n",
      "https://bbs.hupu.com/search?q=文心一言&topicId=&sortby=general&page=8\n",
      "https://bbs.hupu.com/search?q=文心一言&topicId=&sortby=general&page=9\n",
      "https://bbs.hupu.com/search?q=文心一言&topicId=&sortby=general&page=10\n",
      "https://bbs.hupu.com/search?q=通义千问&topicId=&sortby=general&page=1\n",
      "https://bbs.hupu.com/search?q=通义千问&topicId=&sortby=general&page=2\n",
      "https://bbs.hupu.com/search?q=通义千问&topicId=&sortby=general&page=3\n",
      "https://bbs.hupu.com/search?q=通义千问&topicId=&sortby=general&page=4\n",
      "https://bbs.hupu.com/search?q=通义千问&topicId=&sortby=general&page=5\n",
      "https://bbs.hupu.com/search?q=通义千问&topicId=&sortby=general&page=6\n",
      "https://bbs.hupu.com/search?q=通义千问&topicId=&sortby=general&page=7\n",
      "https://bbs.hupu.com/search?q=通义千问&topicId=&sortby=general&page=8\n",
      "https://bbs.hupu.com/search?q=通义千问&topicId=&sortby=general&page=9\n",
      "https://bbs.hupu.com/search?q=通义千问&topicId=&sortby=general&page=10\n",
      "https://bbs.hupu.com/search?q=智谱清言&topicId=&sortby=general&page=1\n",
      "https://bbs.hupu.com/search?q=智谱清言&topicId=&sortby=general&page=2\n",
      "https://bbs.hupu.com/search?q=智谱清言&topicId=&sortby=general&page=3\n",
      "https://bbs.hupu.com/search?q=智谱清言&topicId=&sortby=general&page=4\n",
      "https://bbs.hupu.com/search?q=智谱清言&topicId=&sortby=general&page=5\n",
      "https://bbs.hupu.com/search?q=智谱清言&topicId=&sortby=general&page=6\n",
      "https://bbs.hupu.com/search?q=智谱清言&topicId=&sortby=general&page=7\n",
      "https://bbs.hupu.com/search?q=智谱清言&topicId=&sortby=general&page=8\n",
      "https://bbs.hupu.com/search?q=智谱清言&topicId=&sortby=general&page=9\n",
      "https://bbs.hupu.com/search?q=智谱清言&topicId=&sortby=general&page=10\n",
      "https://bbs.hupu.com/search?q=天工AI&topicId=&sortby=general&page=1\n",
      "https://bbs.hupu.com/search?q=天工AI&topicId=&sortby=general&page=2\n",
      "https://bbs.hupu.com/search?q=天工AI&topicId=&sortby=general&page=3\n",
      "https://bbs.hupu.com/search?q=天工AI&topicId=&sortby=general&page=4\n",
      "https://bbs.hupu.com/search?q=天工AI&topicId=&sortby=general&page=5\n",
      "https://bbs.hupu.com/search?q=天工AI&topicId=&sortby=general&page=6\n",
      "https://bbs.hupu.com/search?q=天工AI&topicId=&sortby=general&page=7\n",
      "https://bbs.hupu.com/search?q=天工AI&topicId=&sortby=general&page=8\n",
      "https://bbs.hupu.com/search?q=天工AI&topicId=&sortby=general&page=9\n",
      "https://bbs.hupu.com/search?q=天工AI&topicId=&sortby=general&page=10\n",
      "https://bbs.hupu.com/search?q=讯飞星火&topicId=&sortby=general&page=1\n",
      "https://bbs.hupu.com/search?q=讯飞星火&topicId=&sortby=general&page=2\n",
      "https://bbs.hupu.com/search?q=讯飞星火&topicId=&sortby=general&page=3\n",
      "https://bbs.hupu.com/search?q=讯飞星火&topicId=&sortby=general&page=4\n",
      "https://bbs.hupu.com/search?q=讯飞星火&topicId=&sortby=general&page=5\n",
      "https://bbs.hupu.com/search?q=讯飞星火&topicId=&sortby=general&page=6\n",
      "https://bbs.hupu.com/search?q=讯飞星火&topicId=&sortby=general&page=7\n",
      "https://bbs.hupu.com/search?q=讯飞星火&topicId=&sortby=general&page=8\n",
      "https://bbs.hupu.com/search?q=讯飞星火&topicId=&sortby=general&page=9\n",
      "https://bbs.hupu.com/search?q=讯飞星火&topicId=&sortby=general&page=10\n",
      "https://bbs.hupu.com/search?q=AI对话助手&topicId=&sortby=general&page=1\n",
      "https://bbs.hupu.com/search?q=AI对话助手&topicId=&sortby=general&page=2\n",
      "https://bbs.hupu.com/search?q=AI对话助手&topicId=&sortby=general&page=3\n",
      "https://bbs.hupu.com/search?q=AI对话助手&topicId=&sortby=general&page=4\n",
      "https://bbs.hupu.com/search?q=AI对话助手&topicId=&sortby=general&page=5\n",
      "https://bbs.hupu.com/search?q=AI对话助手&topicId=&sortby=general&page=6\n",
      "https://bbs.hupu.com/search?q=AI对话助手&topicId=&sortby=general&page=7\n",
      "https://bbs.hupu.com/search?q=AI对话助手&topicId=&sortby=general&page=8\n",
      "https://bbs.hupu.com/search?q=AI对话助手&topicId=&sortby=general&page=9\n",
      "https://bbs.hupu.com/search?q=AI对话助手&topicId=&sortby=general&page=10\n",
      "https://bbs.hupu.com/search?q=AI&topicId=&sortby=general&page=1\n",
      "https://bbs.hupu.com/search?q=AI&topicId=&sortby=general&page=2\n",
      "https://bbs.hupu.com/search?q=AI&topicId=&sortby=general&page=3\n",
      "https://bbs.hupu.com/search?q=AI&topicId=&sortby=general&page=4\n",
      "https://bbs.hupu.com/search?q=AI&topicId=&sortby=general&page=5\n",
      "https://bbs.hupu.com/search?q=AI&topicId=&sortby=general&page=6\n",
      "https://bbs.hupu.com/search?q=AI&topicId=&sortby=general&page=7\n",
      "https://bbs.hupu.com/search?q=AI&topicId=&sortby=general&page=8\n",
      "https://bbs.hupu.com/search?q=AI&topicId=&sortby=general&page=9\n",
      "https://bbs.hupu.com/search?q=AI&topicId=&sortby=general&page=10\n"
     ]
    }
   ],
   "source": [
    "ai_assistants = [\"ChatGPT\", \"Claude\", \"Copilot\", \"Gemini\", \"DeepSeek\", \"豆包\", \"Kimi\", \"文心一言\", \"通义千问\", \"智谱清言\", \"天工AI\", \"讯飞星火\", \"AI对话助手\", \"AI\"]\n",
    "for keyWord in ai_assistants:\n",
    "    ulist = []\n",
    "    main(ulist, keyWord)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
